package com.bigdata.collector.labels;

import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.bigdata.collector.model.Page;
import com.bigdata.collector.util.HttpUtils;

public class Collector4Ifeng extends AbstractCollector {

	public Collector4Ifeng(String seed) {
		super(seed);
	}

	private static Logger logger = Logger
			.getLogger(Collector4Ifeng.class);

	public static void main(String[] args) {
		String seed = "http://baike.finance.ifeng.com/search/c_303/p/";
		AbstractCollector lc = new Collector4Ifeng(seed);
		lc.run();
	}

	public void crawler(String url) {
		int pageNum = 1;
		try {
			String previous = "";
			while (true) {
				String pageUrl = url + pageNum;
				Page page = HttpUtils.fetchHttpResponse(pageUrl);
				if (page.content == null)
					break;
				String content = new String(page.content, "utf-8");
				if (content.equals(previous))
					break;
				previous = content;
				Document doc = Jsoup.parse(content);
				if (!extractTags(doc))
					break;
				pageNum++;
				logger.info("page=" + pageNum);
			}
			logger.info("store=" + store + ",invalid=" + invalid + ",dupicate="
					+ duplicate);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	@Override
	boolean extractTags(Document doc) {
		Elements es = doc.getElementsByAttributeValue("class", "name cblue");
		for (Element e : es) {
			Elements temp = e.getElementsByTag("a");
			String label = temp.first().text().trim();
			super.save2DB(label, "0");
		}
		return true;
	}

	@Override
	public void extractURLs(Document doc) {

	}
}
